package com.shujia.MapReduce;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Hashtable;

public class Demo7MapJoin {
    /**
     * 适用于大表关联小表
     * 原理：将小表广播到每一个MapTask
     * 在MapReduce编程中，一般会使用Hashtable存储小表的数据
     */
    public static class MapJoinMapper extends Mapper<LongWritable, Text, Text, Text> {

        Hashtable<String, String> studentHashTable = null;

        // 每个Map任务会执行一次
        @Override
        protected void setup(Mapper<LongWritable, Text, Text, Text>.Context context) throws IOException {
            studentHashTable = new Hashtable<>();
            URI[] cacheFiles = context.getCacheFiles();
            URI cacheFile = cacheFiles[0];

            // new FileReader默认会读取在task执行所在的节点上的目录
//            BufferedReader br = new BufferedReader(new FileReader(cacheFile.toString()));

            // 构建FileSystem对象 用于获取HDFS上的文件
            FileSystem fs = FileSystem.get(context.getConfiguration());
            FSDataInputStream inputStream = fs.open(new Path(cacheFile.toString()));

            BufferedReader br = new BufferedReader(new InputStreamReader(inputStream));
            String line;
            while ((line = br.readLine()) != null) {
                String[] splits = line.split(",");
                String id = splits[0];
                // 以id作为key，当前行的数据作为value构建HashTable
//                System.out.println(id + "#" + line);
                studentHashTable.put(id, line);
            }

        }

        // 每条数据会执行一次
        @Override
        protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context) throws IOException, InterruptedException {
            String[] splits = value.toString().split(",");
            String id = splits[0];
            String score = splits[2];
//            System.out.println("student's id is " + id);
            // 判断id是否在studentHashTable，如果在说明能关联上
            if (studentHashTable.containsKey(id)) {
//                System.out.println("################" + id + "################");
                String studentInfo = studentHashTable.get(id);
                context.write(new Text(studentInfo), new Text(score));
            }
        }
    }

    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException, URISyntaxException {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        job.setJobName("Demo7MapJoin");
        job.setJarByClass(Demo7MapJoin.class);

        // 配置Map任务
        job.setMapperClass(MapJoinMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);

        // 添加缓存文件
        job.addCacheFile(new URI("/data/student/students.txt"));

        // 配置输入输出路径
        FileInputFormat.addInputPath(job, new Path("/data/score"));
        // 判断输出路径是否存在，存在即删除
        FileSystem fs = FileSystem.get(job.getConfiguration());
        if (fs.exists(new Path("/mapJoinOutput1"))) {
            fs.delete(new Path("/mapJoinOutput1"), true);
        }
        FileOutputFormat.setOutputPath(job, new Path("/mapJoinOutput1"));

        job.waitForCompletion(true);
    }
    /**
     * hadoop jar Hadoop-1.0.jar com.shujia.MapReduce.Demo7MapJoin
     */
}
